from training.msg import ChunkMsg
from iop import BusinessService
from langchain_community.document_loaders import TextLoader
from langchain.text_splitter import CharacterTextSplitter
import json
import qianfan
import numpy as np

class ReadDataSource(BusinessService):
    
    def get_adapter_type():
        
        return "Ens.InboundAdapter"
    
    def on_init(self):
        if not hasattr(self, 'target'):
            # 如果没有，将其设置为 'Instance.Of.SaveInTxtBo' 作为默认值
            self.target = 'SaveVectorOp'
        return
    
    def on_process_input(self, message_input):
        file = '/irisdev/app/data/1.txt'
        loader = TextLoader(file,encoding='utf-8')
        documents = loader.load()
        # self.trace(documents[0].page_content)
        
        text_splitter = CharacterTextSplitter(chunk_size=20, chunk_overlap=0,separator="\n")
        documents = text_splitter.split_documents(documents)
        self.trace(str(type(documents[0])))
        json_data = json.dumps(documents[0].page_content)
        self.trace(json_data)
        msg = ChunkMsg()
        msg.docs = documents
        self.send_request_async(self.target,msg)
        

            